package com.marlowe.asyouknow.utils;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.marlowe.asyouknow.pojo.Content;
import com.marlowe.asyouknow.pojo.HotList;
import com.marlowe.asyouknow.pojo.News;
import com.marlowe.asyouknow.pojo.Pictures;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.net.URL;
import java.util.*;

/**
 * @program: as-you-know
 * @description: html解析工具类
 * @author: Marlowe
 * @create: 2020-12-21 10:30
 **/
@Component
public class HtmlParseUtil {

    public List<Content> parseJD(String keyWords) throws IOException {
        // 获取请求      https://search.jd.com/Search?keyword=java
        String url = "https://search.jd.com/Search?keyword=" + keyWords;
        Document document = Jsoup.parse(new URL(url), 30000);

        // 解析网页。（Jsoup返回Document就是浏览器Document对象）
        Element element = document.getElementById("J_goodsList");

        // 获取所有的li元素
        Elements elements = element.getElementsByTag("li");

        //获取元素中的内容，这里的el就是每一个li标签了

        ArrayList<Content> goodsList = new ArrayList<>();

        for (Element el : elements) {
            String img = el.getElementsByTag("img").eq(0).attr("data-lazy-img");
            String price = el.getElementsByClass("p-price").eq(0).text();
            String title = el.getElementsByClass("p-name").eq(0).text();
            System.out.println("==============================================================");
            System.out.println(img);
            System.out.println(price);
            System.out.println(title);
            Content content = new Content(title, img, price);
            goodsList.add(content);
        }
        return goodsList;
    }

    /**
     * 爬取新闻
     *
     * @return
     * @throws Exception
     */
    public List<News> parseNews() throws Exception {
        /**
         * es 一次性最多插入100M，可以多次，修改话题termid范围，以及page范围
         */
        int count = 0;
        int termId = 8;
        int page = 1000;
        String url = null;
        ArrayList<News> newsList = new ArrayList<>();
        for (int i = 1; i <= 5; i++) {
            for (int j = 0; j <= 1000; j++) {
                //拼接json数据包接口
                url = "http://www.infzm.com/contents?term_id=" + i + "&page=" + j + "&format=json";
                System.out.println("json接口url:" + url);
                String jsonString = null;
                try {
                    jsonString = doGetOhter(url);
                    //将JSON转化为Map
                    Map mapString = (Map) JSON.parse(jsonString);
                    Map data = (Map) mapString.get("data");
                    JSONArray jsonArray = JSONObject.parseArray(data.get("contents").toString());
                    // 当前接口没数据，说明此栏目数据已经提取完，跳到下一termId
                    if (jsonArray == null) {
                        break;
                    }
                    for (int g = 0; g < jsonArray.size(); g++) {
                        JSONObject partDaily = jsonArray.getJSONObject(g);
                        String jsonEntryLinks = partDaily.getString("id");
                        String realUrl = "http://www.infzm.com/contents/" + jsonEntryLinks;
                        Document document = Jsoup.parse(new URL(realUrl), 30000);
                        String contents = "";
                        // 获得标题
                        String title = document.getElementsByTag("h1").text();

                        // 获得正文
                        Elements elements = document.getElementsByClass("nfzm-content__fulltext");
                        for (Element element : elements) {
                            contents += element.getElementsByTag("p").text();
                        }

                        // 去躁
                        if (contents != null && contents.length() != 0 && title != null && title.length() != 0 && !"User not found.".equals(title)) {
                            System.out.println("新闻标题为:" + title);
                            System.out.println("新闻正文为:" + contents);
                            System.out.println("新闻url为:" + realUrl);
                            News news = new News(title, contents, realUrl);
                            newsList.add(news);
                            count++;
                            System.out.println("已爬取" + count + "个网页");
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            System.out.println("=====================================此话题结束=================================================================");
        }
        System.out.println("======================================================================================================");
        System.out.println("爬取结束，共爬取" + count + "个网页");
        System.out.println("爬取列表大小：" + newsList.size());
        return newsList;
    }

    /**
     * 获取网页json字符串
     *
     * @param url
     * @return
     * @throws Exception
     */
    public static String doGetOhter(String url) throws Exception {
        //1,创建一个httpClient对象
        CloseableHttpClient client = HttpClients.createDefault();
        //2,创建uriBuilder 对于httpClient4.3访问指定页面url必须要使用http://开始
        URIBuilder uriBuilder = new URIBuilder(url);
        //4,创建httpget对象
        HttpGet httpGet = new HttpGet(uriBuilder.build());
        //5,设置请求报文头部的编码
        httpGet.setHeader(new BasicHeader("Content-Type", "application/x-www-form-urlencoded; utf-8"));
        //6,设置期望服务返回的编码
        httpGet.setHeader(new BasicHeader("Accept", "text/plain;charset=utf-8"));
        //7，请求服务
        CloseableHttpResponse response = client.execute(httpGet);
        //8，获取请求返回码
        int statusCode = response.getStatusLine().getStatusCode();
        //9如果请求返回码是200，则说明请求成功
        String str = "";
        if (statusCode == 200) {
            //10,获取返回实体
            HttpEntity entity = response.getEntity();
            //11,通过EntityUtils的一个工具类获取返回的内容
            str = EntityUtils.toString(entity);
        } else {
            System.out.println("请求失败！");
        }
        response.close();
        client.close();
        return str;
    }

    /**
     * 爬取百度图库图片
     *
     * @return
     * @throws Exception
     */
    public List<Pictures> parsePictures() throws Exception {

        int pageNumber = 10;
        List<Pictures> pictureList = new ArrayList<>();
        // 关键词列表
        String[] strings = new String[]{"小米", "Apple", "华为", "魅族",
                "王宝强", "冯绍峰", "雷佳音", "成龙", "吴京", "李连杰", "范伟", "周冬雨", "刘亦菲", "迪丽热巴", "古力娜扎", "杨幂", "周星驰", "周润发", "甄子丹", "李易峰", "沈腾", "赵本山",
                "邓紫棋", "徐佳莹", "林俊杰", "张惠妹", "吴莫愁", "薛之谦", "陈奕迅", "张杰", "张学友", "泰勒", "许嵩",
                "奔驰", "宝马", "奥迪", "长安", "特斯拉", "大众", "本田", "蔚来",
                "苹果", "香蕉", "茄子", "番茄", "土豆", "草莓",
                "玫瑰花", "喇叭花", "油菜花", "菊花", "梅花", "竹子",
                "羊", "猪", "狗", "兔子", "猫", "牛"};
        List<String> wordList = new ArrayList<String>(Arrays.asList(strings));

        for (String s : wordList) {
            for (int i = 0; i < pageNumber; i++) {
                String url = "https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord="
                        + s + "&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=&ic=&hd=&latest=&copyright=&word=" + s +
                        "&s=&se=&tab=&width=&height=&face=0&istype=2&qc=&nc=1&fr=&expermode=&force=&pn=" + i * 30 + "&rn=30&gsm=&666=";
                String jsonString = null;
                try {
                    jsonString = doGetOhter(url);
                    //将JSON转化为Map
                    Map mapString = (Map) JSON.parse(jsonString);
                    JSONArray jsonArray = JSONObject.parseArray(mapString.get("data").toString());
                    for (int g = 0; g < jsonArray.size(); g++) {
                        JSONObject partDaily = jsonArray.getJSONObject(g);
                        String jsonEntryLinks = partDaily.getString("thumbURL");
                        String jsonEntryTitle = partDaily.getString("fromPageTitleEnc");
                        if (jsonEntryTitle != null && jsonEntryLinks != null) {
                            System.out.println(jsonEntryLinks);
                            System.out.println(jsonEntryTitle);
                            Pictures picture = new Pictures(jsonEntryTitle, jsonEntryLinks);
                            pictureList.add(picture);
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
                System.out.println("======================================================================================================");
            }
        }
        System.out.println("关键词个数" + wordList.size());
        System.out.println("成功爬取：" + pictureList.size() + "张图片");
        return pictureList;
    }


    /**
     * 获得百度搜索热榜
     *
     * @return
     * @throws Exception
     */
    public List<HotList> parseHotList(String keywords) throws Exception {
        List<HotList> hotLists = new ArrayList<>();

        String url = "https://www.baidu.com/s?&wd=LOL";
        Document document = Jsoup.parse(new URL(url), 30000);
        Elements elements = document.getElementsByClass("toplist1-tr");
        for (Element element : elements) {
            String title = element.getElementsByTag("a").eq(0).attr("title");
            String href = "https://www.baidu.com/s?wd=" + title;
            String number = element.getElementsByClass("toplist1-right-num").eq(0).text();
            HotList hotList = new HotList(title, href, number);
            hotLists.add(hotList);

        }
        System.out.println("百度热榜爬取成功");
        return hotLists;
    }
}
